'''
Created on Dec 7, 2009

@author: wangjo
'''
from google.appengine import runtime
from google.appengine.ext import webapp
from google.appengine.api.labs import taskqueue
from google.appengine.ext.webapp.util import run_wsgi_app
import spider
import utility
import datamodel
import logging

_PER_CRAWLER_MODEL_COUNT = 6
class CronPage(webapp.RequestHandler):
    def get(self, action):
        action = action.lower()
        logging.debug('action: %s'%action)
        if action == 'column':
            self._cronColumn()
        elif action == 'host':
            self._cronHost()
        elif action == 'show':
            self._cronShow()
        elif action == 'sodexo':
            self._cronSodexo()
        else:
            logging.info('unknown action')
            l = datamodel.log(action='unkown action', content = action, successed = False)
            l.put()
        
    def _cronColumn(self):
        s = spider.crawler()
        try:
            s.crawler('http://www.1cbn.com/leadership/', False)
        except runtime.DeadlineExceededError, e:
            logging.error('cron task: crawler model has met deadline exceeded error')
            
    def _cronHost(self):
        hosts = datamodel.log.get_non_crawled_hosts(_PER_CRAWLER_MODEL_COUNT)
        logging.debug('hosts count: %d'%len(hosts))
        s = spider.crawler()
        try:
            for a in hosts:
                s.crawler(a.urlcbn, False)
                datamodel.log.add_crawler_host(a.urlcbn, True)
            if len(hosts) == 0:
                s.crawler('http://www.1cbn.com/cbninfo/EmceeInfo.aspx?EmceeID=1', False)
        except runtime.DeadlineExceededError, e:
            logging.error('cron task: fetch host has met deadline exceeded error:%s'%e.message)
    
    def _cronShow(self):
        columns = datamodel.log.get_non_crawled_shows(_PER_CRAWLER_MODEL_COUNT)
        logging.debug('columns count: %d'%len(columns))
        try:
            for a in columns:
                if not a.radio:
                    spider.fetch_show(a)
        except runtime.DeadlineExceededError, e:
            logging.error('cron task: fetch show has met deadline exceeded error:%s'%e.message)
    
    def _cronSodexo(self):
        logging.info('start cron sodexo')
        baseurl = 'http://sodexo.davyin.com'
        temp = 'http://sodexo.davyin.com/affiliates/search?page=%d'
        try:
            page = spider.get_sodex_pagenum(baseurl)
            logging.info('page:%d'%page)
            taskqueue.add(url='/fetchsodexo', params={'url': baseurl}, method='GET')
            pages = range(1, page)
            for p in pages:
                url = temp%p
                taskqueue.add(url='/fetchsodexo', params={'url': url}, method='GET')
        except runtime.DeadlineExceededError, e:
            logging.error('cron task: fetch sodexo has met deadline exceeded error:%s'%e.message)
            
application = webapp.WSGIApplication([
            (r'/cron/(.*)', CronPage),
            ], debug=utility.DEBUG)

def main():
    run_wsgi_app(application)

if __name__ == "__main__":
    main()